For this snippet, it is necessary use the follow functions From decimal to hexadecimal conversion

function decToHex(IN)
    local B,K,OUT,I,D = 16, "0123456789ABCDEF", "", 0
    while IN > 0 do
        I = I+1
        IN,D = math.floor(IN/B), IN%B+1
        OUT = string.sub(K,D,D)..OUT
    end
    return "#"..OUT
end

And From hexadecimal to rgb conversion

function hexToRGB (hex)
    local hex = hex:gsub("#","")
    if hex:len() == 3 then
        return (tonumber("0x"..hex:sub(1,1))*17)/255, (tonumber("0x"..hex:sub(2,2))*17)/255, (tonumber("0x"..hex:sub(3,3))*17)/255
    else
        return tonumber("0x"..hex:sub(1,2))/255, tonumber("0x"..hex:sub(3,4))/255, tonumber("0x"..hex:sub(5,6))/255
    end
end

Now can defiend two function conversion from rgb to hexadecimal and rgb Scale (1 or 255 as maximun value)

function rgbToHex(r, g, b)
    local min = math.min        -- Faster local var than global var
    r, g, b = min(r,255), min(g,255), min(b,255)
    return decToHex(((r*256)+g)*256+b)
end

function rgbScale(r, g, b, Scale)
    if Scale==1 then
        return r/255, g/255, b/255  -- scale RGB from 0 to 1
    elseif Scale==255 then
        return r*255, g*255, b*255  -- scale RGB from 0 to 255
    else
        return r, g, b
    end
end

Here's how can use it:

local h = rgbToHex(255, 255, 255)
print(h)                        -- printed output: #FFFFFF

local r, g, b = hexToRGB (h)
print(r, g, b)                  -- printed output: 1 1 1
print(rgbScale(r, g, b, 255))   -- printed output: 255 255 255